package com.uusee.crawler.job;

import java.io.InputStream;
import java.util.Date;
import java.util.List;
import java.util.Properties;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.UgcDbWriter;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.video.PlayPageProcessor;
import com.uusee.crawler.pageprocessor.video.UserVideoPageProcessor;
import com.uusee.crawler.util.PageProcessorUtils;
import com.uusee.crawler.xmlwriter.UgcXmlWriter;
import com.uusee.framework.bo.UniversalBo;
import com.uusee.framework.util.query.CriteriaInfo;
import com.uusee.shipshape.gt.model.UploadUser;
import com.uusee.shipshape.sp.model.Ugc;

public class TudouUserVideoCrawlJob {
	private Log log = LogFactory.getLog(TudouUserVideoCrawlJob.class);
	private static final String VIDEO_COUNT_REG="<div class=\"page_content\">.*?<span class=\"action\">\\s+\\(共\\s*([0-9]+)\\s*个视频\\)";
	private static final String USERNAME_REG="<div class=\"page_content\">.*?<div class=\"heading\">\\s+<a class=\"feedButton\" title=\"rss订阅\" href=\"http://rss.tudou.com/userID/[0-9]+\"></a><h2>(.*?) 的视频</h2>";
	private static final String USERBLOG_REG="<address><a href=\".*?\" class=\"homeurl\">(.*?)</a></address>";
	
	private static final int PAGE_SIZE=40;
	//2009年7月23日：最大用户ID不超过5千万
	private int start = 12432151;
	private int end = 12432152;
	
	private FetchHTTP fetch = new FetchHTTP();
	private UserVideoPageProcessor userVideoPageProcessor;
	private PlayPageProcessor playPageProcessor;
	private UgcXmlWriter xmlWriter = new UgcXmlWriter();
	private UgcDbWriter dbWriter = new UgcDbWriter();
	private UniversalBo universalBo;
	
	
	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("applicationContext-ugc.xml");
			TudouUserVideoCrawlJob tudouUserVideoCrawlJob = (TudouUserVideoCrawlJob)acx.getBean("tudouUserVideoCrawlJob");
			tudouUserVideoCrawlJob.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	
	public void doExecute() {
		try {
			StopWatch stopWatch = new StopWatch();
			stopWatch.start();
			int totalCrawlUser = 0;
			int totalPage = 0;
			for (int userId = start; userId < end; userId++) {
				try {
					UploadUser uploadUser = getUploadUser(userId+"");
					if (uploadUser==null) {
						log.info("无效的用户ID：www.tudou.com-"+userId);
						continue;
					}
					//保存用户
					int videoCount = uploadUser.getVideoCount();
					CriteriaInfo ci = new CriteriaInfo();
					ci.eq("userId", uploadUser.getUserId());
					ci.eq("sourceSite", uploadUser.getSourceSite());
					List<UploadUser> uploadUserList = universalBo.getEntitiesByCriteriaInfo(UploadUser.class, ci);
					if (uploadUserList != null && uploadUserList.size() == 1) {
						UploadUser oldUploadUser = uploadUserList.get(0);
						oldUploadUser.setVideoCount(videoCount);
						
						if (videoCount<=0) {
							universalBo.doDelete(UploadUser.class, oldUploadUser.getId());
							log.info("删除没有视频的老用户ID：www.tudou.com-"+userId);
						}else{
							oldUploadUser.setEmptyVideoTimes(0);
							oldUploadUser.setMaxPage(uploadUser.getMaxPage());
							oldUploadUser.setLevel(uploadUser.getLevel());
							oldUploadUser.setLastCrawlDate(uploadUser.getLastCrawlDate());
							universalBo.doMerge(oldUploadUser);
						}
					} else if (videoCount>0) {
						universalBo.doSave(uploadUser);
					} else {
						log.info("没有视频的用户ID：www.tudou.com-"+userId);
					}
					
					
					//抓取视频
					int maxPage = uploadUser.getMaxPage();
					if (maxPage>0) {
						totalCrawlUser++;
						totalPage=totalPage+maxPage;
						log.info("抓取土豆视频(用户:"+userId+";视频页数："+maxPage+")");
						
						for (int page = 1;page<=maxPage;page++) {
							CrawlURI crawURI = new CrawlURI();
							crawURI.setSourceSite("www.tudou.com");
							crawURI.setCharset("gbk");
							crawURI.setCrawlUrl("http://www.tudou.com/home/item_u"+userId+"s0p"+page+".html");

							crawl(crawURI);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
				if (userId%100==0) {
					log.info("抓取用户数:"+(userId-start)+";抓取视频页数："+totalPage+";用时:"+stopWatch.getTime()/1000+"秒");
				}
			}
			stopWatch.stop();
			log.info("抓取结束,用时：" + stopWatch.getTime() / 1000 + "秒。");
		} catch (Exception e) {
			log.error(e);
		}
	}
	
	
	private UploadUser getUploadUser(String userId) {
		UploadUser uploadUser = null;
		try {
			CrawlURI crawlURI = new CrawlURI();
			crawlURI.setCharset("gbk");
			crawlURI.setCrawlUrl("http://www.tudou.com/home/item_u"+userId+"s0p1.html");
			fetch.process(crawlURI);
			int crawlStatus = crawlURI.getCrawlStatus();
			if (crawlStatus == CrawlStatusCodes.FETCH_SUCCESS) {
				String crawlResult = crawlURI.getCrawlResult();
				//分析用户信息
			    String uploadUsername = PageProcessorUtils.getValue(USERNAME_REG, crawlResult);
			    String uploadUserblog = PageProcessorUtils.getValue(USERBLOG_REG, crawlResult);
				if (StringUtils.isNotEmpty(uploadUserblog) && !uploadUserblog.startsWith("http://")) {
					String domain = "www.tudou.com";
					uploadUserblog = "http://" + domain + uploadUserblog;
				}
				String videoCountString = PageProcessorUtils.getValue(VIDEO_COUNT_REG, crawlResult);
				int videoCount = Integer.valueOf(videoCountString);
				
				
				if (StringUtils.isNotEmpty(uploadUsername)) {
					uploadUser = new UploadUser();
					uploadUser.setUserId(userId);
					uploadUser.setUsername(uploadUsername);
					uploadUser.setUserBlogUrl(uploadUserblog);
					uploadUser.setVideoCount(videoCount);
					if (videoCount != 0 ) {
						int maxPage = videoCount%PAGE_SIZE==0?videoCount/PAGE_SIZE:videoCount/PAGE_SIZE+1;
						uploadUser.setMaxPage(maxPage);
					}
					uploadUser.setSourceSite("www.tudou.com");
					if (videoCount > 10000) {
						uploadUser.setLevel(0);
					} else if(videoCount > 1000) {
						uploadUser.setLevel(1);
					} else if(videoCount > 100) {
						uploadUser.setLevel(2);
					} else if(videoCount > 0) {
						uploadUser.setLevel(3);
					} else {
						uploadUser.setLevel(-1);
					}
					uploadUser.setLastCrawlDate(new Date());
					uploadUser.setUpdateUser("uusee-crawler");
					uploadUser.setUpdateDate(new Date());					
				}
			}
		}catch(Exception e){
			uploadUser = null;
		}
		return uploadUser;
	}
	
	public void crawl(CrawlURI crawURI) {
		try {
			fetch.process(crawURI);
			userVideoPageProcessor.process(crawURI);
			if (crawURI.getCrawlStatus() == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				List<CrawlURI> playCrawURIList = (List<CrawlURI>) crawURI.getModel();
				for (CrawlURI playCrawURI:playCrawURIList) {
					Ugc ugc = (Ugc)playCrawURI.getModel();
					if (StringUtils.isEmpty(ugc.getVid())) {
						continue;
					}
					playCrawURI.setCrawlStatus(CrawlStatusCodes.PAGE_PROCESS_SUCCESS);
					//fetch.process(playCrawURI);
					//playPageProcessor.process(playCrawURI);
					dbWriter.process(playCrawURI);
					//System.out.println(ugc.toString());
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}


	public void setStart(int start) {
		this.start = start;
	}


	public void setEnd(int end) {
		this.end = end;
	}


	public void setFetch(FetchHTTP fetch) {
		this.fetch = fetch;
	}


	public void setUserVideoPageProcessor(UserVideoPageProcessor userVideoPageProcessor) {
		this.userVideoPageProcessor = userVideoPageProcessor;
	}


	public void setPlayPageProcessor(PlayPageProcessor playPageProcessor) {
		this.playPageProcessor = playPageProcessor;
	}


	public void setXmlWriter(UgcXmlWriter xmlWriter) {
		this.xmlWriter = xmlWriter;
	}


	public void setDbWriter(UgcDbWriter dbWriter) {
		this.dbWriter = dbWriter;
	}


	public void setUniversalBo(UniversalBo universalBo) {
		this.universalBo = universalBo;
	}
}
